# 阿里云天池-分析美国选民的总统喜好
# Author: Liu Yuanxi
# 请预先安装词云处理包：pip install wordcloud --user

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from wordcloud import WordCloud, ImageColorGenerator

# 读取候选人信息
candidates = pd.read_csv("/Users/liuyuanxi/学习/华为智能基座/huawei-smart-base-learning/美国总统大选预测/数据集/weball20.txt",
                         sep = '|', names = ['CAND_ID', 'CAND_NAME', 'CAND_ICI', 'PTY_CD', 'CAND_PTY_AFFILIATION',
                                                             'TTL_RECEIPTS', 'TRANS_FROM_AUTH', 'TTL_DISB', 'TRANS_TO_AUTH',
                                                             'COH_BOP', 'COH_COP', 'CAND_CONTRIB', 'CAND_LOANS', 'OTHER_LOANS',
                                                             'CAND_LOAN_REPAY', 'OTHER_LOAN_REPAY', 'DEBTS_OWED_BY', 'TTL_INDIV_CONTRIB',
                                                             'CAND_OFFICE_ST', 'CAND_OFFICE_DISTRICT', 'SPEC_ELECTION', 'PRIM_ELECTION',
                                                             'RUN_ELECTION', 'GEN_ELECTION','GEN_ELECTION_PRECENT', 'OTHER_POL_CMTE_CONTRIB',
                                                             'POL_PTY_CONTRIB', 'CVG_END_DT', 'INDIV_REFUNDS', 'CMTE_REFUNDS'])
# 读取候选人和委员会的联系信息
ccl = pd.read_csv("/Users/liuyuanxi/学习/华为智能基座/huawei-smart-base-learning/美国总统大选预测/数据集/ccl.txt",
                  sep = '|', names = ['CAND_ID', 'CAND_ELECTION_YR', 'FEC_ELECTION_YR', 'CMTE_ID', 'CMTE_TP', 'CMTE_DSGN',
                                                 'LINKAGE_ID'])
# 关联两个表的数据
ccl = pd.merge(ccl, candidates)
# 提取所需列的数据
ccl = pd.DataFrame(ccl, columns = ['CMTE_ID', 'CAND_ID', 'CAND_NAME', 'CAND_PTY_AFFILIATION'])
# 查看前十行数据
# print(ccl.head(10))
# 读取个人捐赠数据
itcont = pd.read_csv("/Users/liuyuanxi/学习/华为智能基座/huawei-smart-base-learning/美国总统大选预测/数据集/itcont_2020_20200722_20200820.txt",
                      sep = '|', names = ['CMTE_ID', 'AMNDT_IND', 'RPT_TP', 'TRANSACTION_PGI', 'IMAGE_NUM', 'TRANSACTION_TP', 'ENTITY_TP',
                                          'NAME', 'CITY', 'STATE', 'ZIP_CODE', 'EMPLOYER', 'OCCUPATION', 'TRANSACTION_DT', 'TRANSACTION_AMT',
                                          'OTHER_ID', 'TRAN_ID', 'FILE_NUM', 'MEMO_CD', 'MEMO_TEXT', 'SUB_ID'])
# 合并 ccl 和 itcont 表格
c_itcont = pd.merge(ccl, itcont)
# 提取所需列的数据
c_itcont = pd.DataFrame(c_itcont, columns = [ 'CAND_NAME', 'NAME', 'STATE', 'EMPLOYER', 'OCCUPATION',
                                           'TRANSACTION_AMT', 'TRANSACTION_DT', 'CAND_PTY_AFFILIATION'])
# print(c_itcont.head(10))

# 数据探索与清理
# c_itcont 是规模为 (756205, 8) 的矩阵
# 使用 c_itcont.info() 查看数据信息：
# Int64Index: 756205 entries, 0 to 756204
# Data columns (total 8 columns):
#  #   Column                Non-Null Count   Dtype
# ---  ------                --------------   -----
#  0   CAND_NAME             756205 non-null  object
#  1   NAME                  756205 non-null  object
#  2   STATE                 756160 non-null  object
#  3   EMPLOYER              737413 non-null  object
#  4   OCCUPATION            741294 non-null  object
#  5   TRANSACTION_AMT       756205 non-null  int64
#  6   TRANSACTION_DT        756205 non-null  int64
#  7   CAND_PTY_AFFILIATION  756205 non-null  object
# dtypes: int64(2), object(6)
# memory usage: 51.9+ MB
# STATE, EMPLOYER, OCCUPATION 有缺失值，日期列表为 int64 类型，需要转换为 str 类型
# 缺失值处理，统一填充 NOT_PROVIDED
c_itcont['STATE'].fillna('NOT_PROVIDED', inplace = True)
c_itcont['EMPLOYER'].fillna('NOT_PROVIDED', inplace = True)
c_itcont['OCCUPATION'].fillna('NOT_PROVIDED', inplace = True)
# 对日期进行数据处理
c_itcont['TRANSACTION_DT'] = c_itcont['TRANSACTION_DT'].astype(str)
# 将日期格式改为年、月、日
c_itcont['TRANSACTION_DT'] = [i[3:7]+i[0]+i[1:3] for i in c_itcont['TRANSACTION_DT']]
# print(c_itcont.head(10))
# 使用 c_itcont.describe() 查看表中数据分布情况
#        TRANSACTION_AMT
# count     7.562050e+05
# mean      1.504307e+02
# std       2.320452e+03
# min      -5.600000e+03
# 25%       2.000000e+01
# 50%       3.500000e+01
# 75%       1.000000e+02
# max       1.500000e+06
# 统计美国每个党派获得捐赠的总和排名，取前十名
print(c_itcont.groupby('CAND_PTY_AFFILIATION').sum().sort_values('TRANSACTION_AMT', ascending = False).head(10))
# 统计每个候选人所获的捐赠的总和排名，取前十名
print(c_itcont.groupby("CAND_NAME").sum().sort_values("TRANSACTION_AMT", ascending = False).head(10))
# 统计捐款人职业，按捐款数量排名，取前十名
print(c_itcont.groupby('OCCUPATION').sum().sort_values("TRANSACTION_AMT", ascending = False).head(10))
# 统计每个州获得捐款的总额和排名，取前五名
print(c_itcont.groupby('STATE').sum().sort_values("TRANSACTION_AMT", ascending = False).head(5))
# 统计每个州捐款人数量
print(c_itcont['STATE'].value_counts().head(5))

# 数据可视化
# # 各州 总捐款数/总捐款人数 柱状图
# st_amt = c_itcont.groupby('STATE').sum().sort_values('TRANSACTION_AMT', ascending = False)[:10]
# # st_amt = pd.DataFrame(st_amt, columns = 'TRANSACTION_AMT')
# st_amt.plot(kind = 'bar')
# 拜登捐款人词云图
import os
os.rename('/Users/liuyuanxi/学习/华为智能基座/huawei-smart-base-learning/美国总统大选预测/Trump.jpg', 'Trump.jpg') # 将文件重新命名
data = ' '.join(c_itcont['CAND_NAME'].tolist())
bg = plt.imread("Trump.jpg")
# 生成
wc = WordCloud(# FFFAE3
    background_color="white",  # 设置背景为白色，默认为黑色
    width=890,  # 设置图片的宽度
    height=600,  # 设置图片的高度
    mask=bg,    # 画布
    margin=10,  # 设置图片的边缘
    max_font_size=100,  # 显示的最大的字体大小
    random_state=20,  # 为每个单词返回一个PIL颜色
).generate_from_text(data)
# 图片背景
bg_color = ImageColorGenerator(bg)
# 开始画图
plt.imshow(wc.recolor(color_func=bg_color))
# 为云图去掉坐标轴
plt.axis("off")
# 画云图，显示
# 保存云图
wc.to_file("Trump_wordcloud.png")